
<!DOCTYPE HTML>
<html lang="" >
    <head>
        <meta charset="UTF-8">
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <title>3.1 使用正则的爬虫 · GitBook</title>
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <meta name="description" content="">
        <meta name="generator" content="GitBook 3.2.3">
        
        
        
    
    <link rel="stylesheet" href="../gitbook/style.css">

    
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
                
            
        

    

    
        
    
        
    
        
    
        
    
        
    
        
    

        
    
    
    <meta name="HandheldFriendly" content="true"/>
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
    <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">

    
    <link rel="next" href="xpath与XML.html" />
    
    
    <link rel="prev" href="结构化与非结构化.html" />
    

    </head>
    <body>
        
<div class="book">
    <div class="book-summary">
        
            
<div id="book-search-input" role="search">
    <input type="text" placeholder="Type to search" />
</div>

            
                <nav role="navigation">
                


<ul class="summary">
    
    

    

    
        
        
    
        <li class="chapter " data-level="1.1" data-path="../">
            
                <a href="../">
            
                    
                    前言
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="正则表达式.html">
            
                <a href="正则表达式.html">
            
                    
                    第一章：正则表达式
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.2.1" data-path="正则表达式入门.html">
            
                <a href="正则表达式入门.html">
            
                    
                    1.1 正则表达式入门
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.2" data-path="Python中的正则表达式.html">
            
                <a href="Python中的正则表达式.html">
            
                    
                    1.2 Python中的正则表达式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.3" data-path="基本正则表达式.html">
            
                <a href="基本正则表达式.html">
            
                    
                    1.3 基本正则表达式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.4" data-path="分组.html">
            
                <a href="分组.html">
            
                    
                    1.4 分组
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.5" data-path="先行断言.html">
            
                <a href="先行断言.html">
            
                    
                    1.5 先行断言
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.6" data-path="标记.html">
            
                <a href="标记.html">
            
                    
                    1.6 标记
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.7" data-path="替换与正则表达式编译.html">
            
                <a href="替换与正则表达式编译.html">
            
                    
                    1.7 替换与正则编译
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.8" data-path="正则表达式练习.html">
            
                <a href="正则表达式练习.html">
            
                    
                    1.8 正则表达式练习
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="Python爬虫学习目标.html">
            
                <a href="Python爬虫学习目标.html">
            
                    
                    第二章：爬虫与Requests
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.3.1" data-path="Requests的简单使用.html">
            
                <a href="Requests的简单使用.html">
            
                    
                    2.1 Requests的简单使用
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.2" data-path="深入了解Requests.html">
            
                <a href="深入了解Requests.html">
            
                    
                    2.2 深入了解Requets
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.4" data-path="结构化与非结构化.html">
            
                <a href="结构化与非结构化.html">
            
                    
                    第三章：非结构化与结构化数据提取
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter active" data-level="1.4.1" data-path="正则抓取.html">
            
                <a href="正则抓取.html">
            
                    
                    3.1 使用正则的爬虫
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.2" data-path="xpath与XML.html">
            
                <a href="xpath与XML.html">
            
                    
                    3.2 Xpath与XML类库
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.3" data-path="lxml.html">
            
                <a href="lxml.html">
            
                    
                    3.3 LXML库的使用
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.4" data-path="Xpath爬虫案例.html">
            
                <a href="Xpath爬虫案例.html">
            
                    
                    3.4 案例：贴吧图片
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.5" data-path="Json与JsonPath.html">
            
                <a href="Json与JsonPath.html">
            
                    
                    3.5 Json与Json数据获取
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.6" data-path="实例：糗事百科网站抓取.html">
            
                <a href="实例：糗事百科网站抓取.html">
            
                    
                    3.6 糗事百科网站爬虫
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    

    

    <li class="divider"></li>

    <li>
        <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
            Published with GitBook
        </a>
    </li>
</ul>


                </nav>
            
        
    </div>

    <div class="book-body">
        
            <div class="body-inner">
                
                    

<div class="book-header" role="navigation">
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href=".." >3.1 使用正则的爬虫</a>
    </h1>
</div>




                    <div class="page-wrapper" tabindex="-1" role="main">
                        <div class="page-inner">
                            
<div id="book-search-results">
    <div class="search-noresults">
    
                                <section class="normal markdown-section">
                                
                                <h2 id="&#x4F7F;&#x7528;&#x6B63;&#x5219;&#x8868;&#x8FBE;&#x5F0F;&#x7684;&#x722C;&#x866B;&#x6293;&#x53D6;&#x6570;&#x636E;">&#x4F7F;&#x7528;&#x6B63;&#x5219;&#x8868;&#x8FBE;&#x5F0F;&#x7684;&#x722C;&#x866B;&#x6293;&#x53D6;&#x6570;&#x636E;</h2>
<p>&#x76EE;&#x6807;&#x7F51;&#x7AD9;&#xFF1A; <a href="http://www.neihan8.com/article/list_5_1.html" target="_blank">http://www.neihan8.com/article/list_5_1.html</a></p>
<p>&#x6253;&#x5F00;&#x4E4B;&#x540E;&#xFF0C;&#x4E0D;&#x96BE;&#x770B;&#x5230;&#x91CC;&#x9762;&#x4E00;&#x4E2A;&#x4E00;&#x4E2A;&#x7070;&#x5E38;&#x6709;&#x5185;&#x6DB5;&#x7684;&#x6BB5;&#x5B50;&#xFF0C;&#x5F53;&#x4F60;&#x8FDB;&#x884C;&#x7FFB;&#x9875;&#x7684;&#x65F6;&#x5019;&#xFF0C;&#x6CE8;&#x610F;url&#x5730;&#x5740;&#x7684;&#x53D8;&#x5316;&#xFF1A;</p>
<p>&#x7B2C;&#x4E00;&#x9875;url: http: //www.neihan8.com/article/list_5_1 .html</p>
<p>&#x7B2C;&#x4E8C;&#x9875;url: http: //www.neihan8.com/article/list_5_2 .html</p>
<p>&#x7B2C;&#x4E09;&#x9875;url: http: //www.neihan8.com/article/list_5_3 .html</p>
<p>&#x7B2C;&#x56DB;&#x9875;url: http: //www.neihan8.com/article/list_5_4 .html</p>
<p>&#x8FD9;&#x6837;&#x6211;&#x4EEC;&#x7684;url&#x89C4;&#x5F8B;&#x627E;&#x5230;&#x4E86;&#xFF0C;&#x8981;&#x60F3;&#x722C;&#x53D6;&#x6240;&#x6709;&#x7684;&#x6BB5;&#x5B50;&#xFF0C;&#x53EA;&#x9700;&#x8981;&#x4FEE;&#x6539;&#x4E00;&#x4E2A;&#x53C2;&#x6570;&#x5373;&#x53EF;&#x3002; &#x4E0B;&#x9762;&#x6211;&#x4EEC;&#x5C31;&#x5F00;&#x59CB;&#x4E00;&#x6B65;&#x4E00;&#x6B65;&#x5C06;&#x6240;&#x6709;&#x7684;&#x6BB5;&#x5B50;&#x722C;&#x53D6;&#x4E0B;&#x6765;&#x5427;&#x3002;</p>
<h2 id="&#x83B7;&#x53D6;&#x6570;&#x636E;">&#x83B7;&#x53D6;&#x6570;&#x636E;</h2>
<pre><code class="lang-python"><span class="hljs-keyword">import</span> requests

url = <span class="hljs-string">&apos;http://www.neihan8.com/article/list_5_1.html&apos;</span>
headers = {
    <span class="hljs-string">&quot;User-Agent&quot;</span>: <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36&quot;</span>,
     <span class="hljs-string">&quot;Accept-Encoding&quot;</span>:<span class="hljs-keyword">None</span>,
    <span class="hljs-string">&quot;Accept-Language&quot;</span>: <span class="hljs-string">&quot;zh-CN,zh;q=0.8&quot;</span>
}
info = requests.get(url,headers=headers)
print(info.content.decode(<span class="hljs-string">&apos;gbk&apos;</span>))
</code></pre>
<p>&#x7A0B;&#x5E8F;&#x6B63;&#x5E38;&#x6267;&#x884C;&#x7684;&#x8BDD;&#xFF0C;&#x6211;&#x4EEC;&#x4F1A;&#x5728;&#x5C4F;&#x5E55;&#x4E0A;&#x6253;&#x5370;&#x4E86;&#x5185;&#x6DB5;&#x6BB5;&#x5B50;&#x7B2C;&#x4E00;&#x9875;&#x7684;&#x5168;&#x90E8;html&#x4EE3;&#x7801;&#x3002;</p>
<h2 id="&#x5220;&#x9009;&#x6570;&#x636E;">&#x5220;&#x9009;&#x6570;&#x636E;</h2>
<p>&#x83B7;&#x53D6;&#x5230;&#x4E86;HTML&#x5185;&#x5BB9;&#x4E4B;&#x540E;&#xFF0C;&#x53EF;&#x4EE5;&#x53D1;&#x73B0;&#x6709;&#x5F88;&#x591A;&#x5185;&#x5BB9;&#x662F;&#x4E0D;&#x9700;&#x8981;&#x7684;&#xFF0C;&#x6211;&#x4EEC;&#x9700;&#x8981;&#x4F7F;&#x7528;&#x6B63;&#x5219;&#x6765;&#x6E05;&#x6D17;&#x6389;&#x4E0D;&#x9700;&#x8981;&#x7684;&#x6570;&#x636E;&#x3002;</p>
<p>&#x9996;&#x5148;&#x6211;&#x4EEC;&#x9700;&#x8981;&#x627E;&#x5230;&#x5339;&#x914D;&#x89C4;&#x5219;&#xFF1A;</p>
<p>&#x6211;&#x4EEC;&#x53EF;&#x4EE5;&#x6253;&#x5F00;&#x5185;&#x6DB5;&#x6BB5;&#x5B50;&#x7684;&#x7F51;&#x9875;&#xFF0C;&#x9F20;&#x6807;&#x70B9;&#x51FB;&#x53F3;&#x952E; &quot; &#x67E5;&#x770B;&#x6E90;&#x4EE3;&#x7801; &quot; &#x4F60;&#x4F1A;&#x53D1;&#x73B0;&#x6BCF;&#x6761;&#x6BB5;&#x5B50;&#x7684;&#x5185;&#x5BB9;&#x5927;&#x81F4;&#x5982;&#x4E0B;</p>
<pre><code class="lang-html"><span class="hljs-tag">&lt;<span class="hljs-name">a</span> <span class="hljs-attr">href</span>=<span class="hljs-string">&quot;/article/44959.html&quot;</span>&gt;</span><span class="hljs-tag">&lt;<span class="hljs-name">b</span>&gt;</span>&#x56DE;&#x5BB6;&#x5954;&#x4E27;<span class="hljs-tag">&lt;/<span class="hljs-name">b</span>&gt;</span><span class="hljs-tag">&lt;/<span class="hljs-name">a</span>&gt;</span><span class="hljs-tag">&lt;/<span class="hljs-name">h4</span>&gt;</span>
                                                                <span class="hljs-tag">&lt;<span class="hljs-name">div</span> <span class="hljs-attr">class</span>=<span class="hljs-string">&quot;f18 mb20&quot;</span>&gt;</span>
                                                                                &#x3000;&#x3000;&#x4E00;&#x8001;&#x592A;&#x592A;&#x8DCB;&#x5C71;&#x6D89;&#x6C34;&#x6765;&#x5230;&#x90E8;&#x961F;&#xFF0C;&#x770B;&#x671B;&#x5979;&#x7684;&#x5B59;&#x5B50;&#xFF0C;<span class="hljs-tag">&lt;<span class="hljs-name">br</span> /&gt;</span>
&#x3000;&#x3000;&#x8B66;&#x536B;&#x95EE;&#xFF1A;&#x201C;&#x5979;&#x627E;&#x8C01;&#xFF1F;&#x201D;&#x8001;&#x592A;&#x8BF4;&#xFF1A;&#x201C;&#x627E;xx&#xFF0C;&#x201D;&#x8B66;&#x536B;&#x6253;&#x5B8C;&#x7535;&#x8BDD;&#x8BF4;&#xFF1A;<span class="hljs-tag">&lt;<span class="hljs-name">br</span> /&gt;</span>
&#x3000;&#x3000;&#x201C;xx&#x4E09;&#x5929;&#x524D;&#x8BF4;&#x5979;&#x4ED6;&#x5976;&#x5976;&#x8FC7;&#x4E16;&#xFF0C;&#x56DE;&#x5BB6;&#x5954;&#x4E27;&#x53BB;&#x4E86;&#xFF0C;&#x5954;&#x4E27;&#x53BB;&#x4E86;&#xFF0C;&#x53BB;&#x4E86;&#x3002;&#x3002;&#x201D;

                                                                <span class="hljs-tag">&lt;/<span class="hljs-name">div</span>&gt;</span>
</code></pre>
<p>&#x6839;&#x636E;HTML&#x6587;&#x672C;&#x7F16;&#x5199;&#x5339;&#x914D;&#x7684;&#x6B63;&#x5219;&#x5982;&#x4E0B;&#xFF1A;</p>
<pre><code class="lang-python">html = info.content.decode(<span class="hljs-string">&apos;gbk&apos;</span>)
data = re.findall(<span class="hljs-string">r&apos;&lt;a\shref=&quot;\/article\/\d+\.html&quot;&gt;(.*?)&lt;/a&gt;.*?&lt;div\sclass=&quot;f18 mb20&quot;&gt;(.*?)&lt;/div&gt;&apos;</span>,html,re.S)

print(data)
</code></pre>
<p>&#x8FD0;&#x884C;&#x4EE3;&#x7801;&#x4F1A;&#x53D1;&#x73B0;&#x8F93;&#x51FA;&#x7684;&#x90E8;&#x5206;&#x7ED3;&#x679C;&#x5982;&#x4E0B;&#xFF1A;</p>
<pre><code>[(&apos;&#x8001;&#x5E08;:&#x5317;&#x5B8B;&#x6709;&#x4E2A;&#x4EBA;&#x548C;&#x4F60;&#x4E00;&#x6837;&#xFF0C;&#x4ED6;&#x59D3;&#x6B66;&#xFF01;&apos;, &apos;\r\n                            &lt;p&gt;\r\n\t\u3000\u3000&#x8001;&#x5E08;:&amp;ldquo;&#x5C0F;&#x660E;&#xFF0C;&#x4F60;&#x7684;&#x68A6;&#x60F3;&#x662F;&#x4EC0;&#x4E48;&#xFF1F;&amp;rdquo;&#x5C0F;&#x660E;&#x6C89;&#x601D;&#x7247;&#x523B;&#x9053;:&amp;ldquo;&#x6709;&#x623F;&#x6709;&#x94FA;&#xFF0C;&#x81EA;&#x5DF1;&#x5F53;&#x8001;&#x677F;&#xFF0C;&lt;br /&gt;\r\n\t
---
</code></pre><p>&#x53EF;&#x4EE5;&#x53D1;&#x73B0;&#x6709;&#x5F88;&#x591A;&#x65E0;&#x610F;&#x4E49;&#x7684;&#x5B57;&#x7B26;&#xFF1A;<code>\r\n\t\u3000</code>&#xFF0C;&#x6240;&#x4EE5;&#x63A5;&#x4E0B;&#x6765;&#x9700;&#x8981;&#x8FDB;&#x884C;&#x66FF;&#x6362;&#x3002;</p>
<pre><code class="lang-python"><span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> data:
    <span class="hljs-keyword">for</span> j <span class="hljs-keyword">in</span> i:
        j = re.sub(<span class="hljs-string">r&apos;[\\r|\\n|\s|&lt;b&gt;|&lt;/b&gt;|&lt;br&gt;|&lt;br /&gt;|&lt;p&gt;|&lt;/p&gt;|\\u3000]&apos;</span>,<span class="hljs-string">&apos;&apos;</span>,j)
        j = j.replace(<span class="hljs-string">&quot;&amp;dqo;&quot;</span>,<span class="hljs-string">&apos;&quot;&apos;</span>).replace(<span class="hljs-string">&quot;&amp;ldqo;&quot;</span>,<span class="hljs-string">&apos;&quot;&apos;</span>).replace(<span class="hljs-string">&quot;&amp;helli;&quot;</span>,<span class="hljs-string">&apos;...&apos;</span>)
        print(j)
</code></pre>
<p>&#x8F93;&#x51FA;&#x7684;&#x90E8;&#x5206;&#x6570;&#x636E;&#x5982;&#x4E0B;&#xFF1A;</p>
<pre><code>&#x8001;&#x5E08;:&#x5317;&#x5B8B;&#x6709;&#x4E2A;&#x4EBA;&#x548C;&#x4F60;&#x4E00;&#x6837;&#xFF0C;&#x4ED6;&#x59D3;&#x6B66;&#xFF01;
&#x8001;&#x5E08;:&quot;&#x5C0F;&#x660E;&#xFF0C;&#x4F60;&#x7684;&#x68A6;&#x60F3;&#x662F;&#x4EC0;&#x4E48;&#xFF1F;&quot;&#x5C0F;&#x660E;&#x6C89;&#x601D;&#x7247;&#x523B;&#x9053;:&quot;&#x6709;&#x623F;&#x6709;&#x94FA;&#xFF0C;&#x81EA;&#x5DF1;&#x5F53;&#x8001;&#x677F;&#xFF0C;&#x59BB;&#x5B50;&#x8C8C;&#x7F8E;&#x5982;&#x82B1;&#xFF0C;&#x8FD8;&#x6709;&#x5F53;&#x5B98;&#x7684;&#x5144;&#x5F1F;&quot;&#x8001;&#x5E08;:&#x5317;&#x5B8B;&#x6709;&#x4E2A;&#x4EBA;&#x548C;&#x4F60;&#x4E00;&#x6837;&#xFF0C;&#x4ED6;&#x59D3;&#x6B66;&#xFF01;
&#x5979;&#x59D0;&#x5C45;&#x7136;&#x662F;&#x6211;&#x521D;&#x4E2D;&#x65F6;&#x5019;&#x7684;&#x521D;&#x604B;&#x5973;&#x53CB;
</code></pre><p>&#x53EF;&#x4EE5;&#x53D1;&#x73B0;&#xFF0C;&#x6211;&#x4EEC;&#x5DF2;&#x7ECF;&#x628A;&#x9700;&#x8981;&#x7684;&#x6570;&#x636E;&#x83B7;&#x53D6;&#x5230;&#x4E86;&#xFF0C;&#x63A5;&#x4E0B;&#x6765;&#x9700;&#x8981;&#x5C06;&#x6570;&#x636E;&#x4FDD;&#x5B58;&#x3002;</p>
<h2 id="&#x4FDD;&#x5B58;&#x6570;&#x636E;">&#x4FDD;&#x5B58;&#x6570;&#x636E;</h2>
<p>&#x8BB2;&#x722C;&#x866B;&#x83B7;&#x53D6;&#x7684;&#x6570;&#x636E;&#x901A;&#x8FC7;&#x4FDD;&#x5B58;&#x81F3;txt&#x6587;&#x4EF6;&#x4E2D;&#xFF1A;</p>
<pre><code class="lang-python"><span class="hljs-keyword">with</span> open(<span class="hljs-string">&apos;./info.txt&apos;</span>,<span class="hljs-string">&apos;a+&apos;</span>) <span class="hljs-keyword">as</span> file:
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> data:
        temp = []
        <span class="hljs-keyword">for</span> j <span class="hljs-keyword">in</span> i:
            j = re.sub(<span class="hljs-string">r&apos;[\\r|\\n|\s|&lt;b&gt;|&lt;/b&gt;|&lt;br&gt;|&lt;br /&gt;|&lt;p&gt;|&lt;/p&gt;|\\u3000]&apos;</span>,<span class="hljs-string">&apos;&apos;</span>,j)
            j = j.replace(<span class="hljs-string">&quot;&amp;dqo;&quot;</span>,<span class="hljs-string">&apos;&quot;&apos;</span>).replace(<span class="hljs-string">&quot;&amp;ldqo;&quot;</span>,<span class="hljs-string">&apos;&quot;&apos;</span>).replace(<span class="hljs-string">&quot;&amp;helli;&quot;</span>,<span class="hljs-string">&apos;...&apos;</span>)
            temp.append(j)
        <span class="hljs-comment">#print(temp)</span>
        file.write(temp[<span class="hljs-number">0</span>] + <span class="hljs-string">&apos;\n&apos;</span> + temp[<span class="hljs-number">1</span>] + <span class="hljs-string">&apos;\n&apos;</span>)
        file.write(<span class="hljs-string">&apos;- - - - - - - - - - - - - - \n&apos;</span>)
</code></pre>
<h2 id="&#x6700;&#x7EC8;&#x4EE3;&#x7801;">&#x6700;&#x7EC8;&#x4EE3;&#x7801;</h2>
<p>&#x7EFC;&#x4E0A;&#xFF0C;&#x6211;&#x4EEC;&#x53EF;&#x4EE5;&#x5C06;&#x5E38;&#x7528;&#x7684;&#x65B9;&#x6CD5;&#x5C01;&#x88C5;&#xFF0C;&#x8FD9;&#x5C31;&#x662F;&#x4E00;&#x4E2A;&#x6BD4;&#x8F83;&#x7CBE;&#x7B80;&#x7684;&#x722C;&#x866B;&#x5C0F;&#x7A0B;&#x5E8F;&#x4E86;&#x3002;</p>
<pre><code class="lang-python"><span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">Duanzi_spider</span><span class="hljs-params">()</span>:</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self)</span>:</span>
        self.url = <span class="hljs-string">&quot;http://www.neihan8.com/article/list_5_%s.html&quot;</span>
        self.headers = {
            <span class="hljs-string">&quot;User_Agent&quot;</span>:<span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleW\
            ebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&quot;</span>,
            <span class="hljs-string">&quot;Accept-Encoding&quot;</span>:<span class="hljs-keyword">None</span>,
            <span class="hljs-string">&quot;Accept-Language&quot;</span>: <span class="hljs-string">&quot;zh-CN,zh;q=0.8&quot;</span>

        }

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">load_page</span><span class="hljs-params">(self,url)</span>:</span>
            <span class="hljs-string">&apos;&apos;&apos;&#x53EF;&#x4EE5;&#x590D;&#x7528;&#x7684;&#x9875;&#x9762;&#x8BF7;&#x6C42;&#x65B9;&#x6CD5;
            &apos;&apos;&apos;</span>
            response = requests.get(url,timeout=<span class="hljs-number">10</span>,headers=self.headers)
            <span class="hljs-keyword">if</span> response.status_code==<span class="hljs-number">200</span>:
                    print(response.request.headers)
                    <span class="hljs-keyword">return</span> response.content.decode(<span class="hljs-string">&quot;gbk&quot;</span>)
            <span class="hljs-keyword">else</span>:
                    <span class="hljs-keyword">raise</span> ValueError(<span class="hljs-string">&quot;status_code is:&quot;</span>,response.status_code)
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_content</span><span class="hljs-params">(self,html)</span>:</span>
        <span class="hljs-string">&apos;&apos;&apos;  &#x6839;&#x636E;&#x7F51;&#x9875;&#x5185;&#x5BB9;&#xFF0C;&#x540C;&#x65F6;&#x5339;&#x914D;&#x6807;&#x9898;&#x548C;&#x6BB5;&#x5B50;&#x5185;&#x5BB9;
        &apos;&apos;&apos;</span>
        pattern = re.compile(<span class="hljs-string">r&apos;&lt;a\shref=&quot;/article/\d+\.html&quot;&gt;(.*?)&lt;/a&gt;.*?&lt;div\sclass=&quot;f18 mb20&quot;&gt;(.*?)&lt;/div&gt;&apos;</span>, re.S)
        t = pattern.findall(html)
        result = []
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> t:
            temp = []
            <span class="hljs-keyword">for</span> j <span class="hljs-keyword">in</span> i:
                    j = re.sub(<span class="hljs-string">r&quot;[&lt;b&gt;|&lt;/b&gt;|&lt;br /&gt;|&lt;br&gt;|&lt;p&gt;|&lt;/p&gt;|\\u3000|\\r\\n|\s]&quot;</span>,<span class="hljs-string">&quot;&quot;</span>,j)
                    j = j.replace(<span class="hljs-string">&quot;&amp;ldqo;&quot;</span>,<span class="hljs-string">&apos;&quot;&apos;</span>).replace(<span class="hljs-string">&quot;&amp;helli;&quot;</span>,<span class="hljs-string">&quot;...&quot;</span>).replace(<span class="hljs-string">&quot;&amp;dqo;&quot;</span>,<span class="hljs-string">&apos;&quot;&apos;</span>).strip()
                    temp.append(j)
            result.append(temp)
    <span class="hljs-keyword">return</span> result

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">save_content</span><span class="hljs-params">(self,content)</span>:</span>
        myFile = open(<span class="hljs-string">&quot;./duanzi.txt&quot;</span>, <span class="hljs-string">&apos;a&apos;</span>)
        <span class="hljs-keyword">for</span> temp <span class="hljs-keyword">in</span> content:
            myFile.write(<span class="hljs-string">&quot;\n&quot;</span>+temp[<span class="hljs-number">0</span>]+<span class="hljs-string">&quot;\n&quot;</span>+temp[<span class="hljs-number">1</span>]+<span class="hljs-string">&quot;\n&quot;</span>)
            myFile.write(<span class="hljs-string">&quot;-----------------------------------------------------&quot;</span>)
        myFile.close()
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">run</span><span class="hljs-params">(self)</span>:</span>
        i = <span class="hljs-number">1</span>
        <span class="hljs-keyword">while</span> <span class="hljs-keyword">True</span>:
                html = self.load_page(self.url%i)
                result = self.get_content(html)
                <span class="hljs-keyword">print</span> (<span class="hljs-string">&quot;&#x6309;&#x56DE;&#x8F66;&#x7EE7;&#x7EED;...&quot;</span>)
                <span class="hljs-keyword">print</span> (<span class="hljs-string">&quot;&#x8F93;&#x5165; quit &#x9000;&#x51FA;&quot;</span>)
                command = input()
                <span class="hljs-keyword">if</span> (command == <span class="hljs-string">&quot;quit&quot;</span>):
                        <span class="hljs-keyword">break</span>
                i+=<span class="hljs-number">1</span>
</code></pre>
<h2 id="&#x7EC3;&#x4E60;">&#x7EC3;&#x4E60;</h2>
<p><strong>&#x83B7;&#x53D6; <a href="http://36kr.com/" target="_blank">http://36kr.com/</a> &#x7F51;&#x7AD9;&#x9996;&#x9875;&#x7684;&#x6240;&#x6709;&#x65B0;&#x95FB;</strong></p>

                                
                                </section>
                            
    </div>
    <div class="search-results">
        <div class="has-results">
            
            <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
            <ul class="search-results-list"></ul>
            
        </div>
        <div class="no-results">
            
            <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
            
        </div>
    </div>
</div>

                        </div>
                    </div>
                
            </div>

            
                
                <a href="结构化与非结构化.html" class="navigation navigation-prev " aria-label="Previous page: 第三章：非结构化与结构化数据提取">
                    <i class="fa fa-angle-left"></i>
                </a>
                
                
                <a href="xpath与XML.html" class="navigation navigation-next " aria-label="Next page: 3.2 Xpath与XML类库">
                    <i class="fa fa-angle-right"></i>
                </a>
                
            
        
    </div>

    <script>
        var gitbook = gitbook || [];
        gitbook.push(function() {
            gitbook.page.hasChanged({"page":{"title":"3.1 使用正则的爬虫","level":"1.4.1","depth":2,"next":{"title":"3.2 Xpath与XML类库","level":"1.4.2","depth":2,"path":"notes/xpath与XML.md","ref":"notes/xpath与XML.md","articles":[]},"previous":{"title":"第三章：非结构化与结构化数据提取","level":"1.4","depth":1,"path":"notes/结构化与非结构化.md","ref":"notes/结构化与非结构化.md","articles":[{"title":"3.1 使用正则的爬虫","level":"1.4.1","depth":2,"path":"notes/正则抓取.md","ref":"notes/正则抓取.md","articles":[]},{"title":"3.2 Xpath与XML类库","level":"1.4.2","depth":2,"path":"notes/xpath与XML.md","ref":"notes/xpath与XML.md","articles":[]},{"title":"3.3 LXML库的使用","level":"1.4.3","depth":2,"path":"notes/lxml.md","ref":"notes/lxml.md","articles":[]},{"title":"3.4 案例：贴吧图片","level":"1.4.4","depth":2,"path":"notes/Xpath爬虫案例.md","ref":"notes/Xpath爬虫案例.md","articles":[]},{"title":"3.5 Json与Json数据获取","level":"1.4.5","depth":2,"path":"notes/Json与JsonPath.md","ref":"notes/Json与JsonPath.md","articles":[]},{"title":"3.6 糗事百科网站爬虫","level":"1.4.6","depth":2,"path":"notes/实例：糗事百科网站抓取.md","ref":"notes/实例：糗事百科网站抓取.md","articles":[]}]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"notes/正则抓取.md","mtime":"2019-05-04T08:39:48.019Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-08T02:50:45.307Z"},"basePath":"..","book":{"language":""}});
        });
    </script>
</div>

        
    <script src="../gitbook/gitbook.js"></script>
    <script src="../gitbook/theme.js"></script>
    
        
        <script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-search/search.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
        
    

    </body>
</html>

